/**
 * 2017年5月17日
 */
package cn.edu.bjtu.workbench.tokenization;

import java.io.InputStream;

import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;


/**
 * 
 * @author Alex
 *
 */
public class AnsjTokenzierFactory extends DefaultTokenizerFactory{
	boolean onlyNoun = false;
	public AnsjTokenzierFactory(){
		
	}
	public AnsjTokenzierFactory(boolean onlyNoun){
		this.onlyNoun = onlyNoun;
	}
	@Override
	public Tokenizer create(String toTokenize) {
		Tokenizer t = onlyNoun ? new OnlyNounAnsjTokenizer(toTokenize) :new AnsjTokenizer(toTokenize);
		if(this.getTokenPreProcessor()!=null){
			t.setTokenPreProcessor(getTokenPreProcessor());
		}
		return t;
	}

	@Override
	public Tokenizer create(InputStream toTokenize) {
		Tokenizer t = onlyNoun ? new OnlyNounAnsjTokenizer(toTokenize) :new AnsjTokenizer(toTokenize);
		if(this.getTokenPreProcessor()!=null){
			t.setTokenPreProcessor(getTokenPreProcessor());
		}
		return t;
	}

}
